#!/usr/bin/python3
# -*- coding: utf-8 -*-
import random
import telnetlib
import time

import httpx
from lxml import etree


def requestUrl(index):
    src_url = 'http://www.xiladaili.com/gaoni/' + str(index)
    url = src_url + str(index)
    if index == 0:
        url = src_url
    # 定义请求头
    headers = {
        "Host": "www.xiladaili.com",
        "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.110 Safari/537.36",
        "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9",
        "Accept-Language": "zh-CN,zh;q=0.9",
        "Accept-Encoding": "gzip, deflate",
        "Referer": "http://www.xiladaili.com/gaoni/",
    }
    client = httpx.Client(http2=True, verify=False)
    response = client.get(url=url, headers=headers)
    return response.content


def parseProxyIpList(content):
    ip_list = []

    # element = etree.parse("./html/xiladaili.html", etree.HTMLParser())
    # print('element= ', element)
    text = str(content, 'utf-8')
    # print(text)
    element = etree.HTML(text)
    # print(element)
    # print(type(element), element, sep='|')
    tbody_tr_td = element.xpath('//table/tbody/tr/td[1]/text()')
    # print(type(tbody_tr), tbody_tr, len(tbody_tr), sep='-')
    for tbody_tr_td_item in tbody_tr_td:
        # print(type(tbody_tr_td_item), tbody_tr_td_item, sep='|')
        ip_temp = 'http://' + tbody_tr_td_item
        print('发现ip：%s' % ip_temp)
        ip_list.append(ip_temp)
    return ip_list


# 过滤有效的ip信息
def filterValidProxyIp(ip_list):
    print('开始过滤可用ip 。。。')
    validList = []
    for ip in ip_list:
        string = ip.split(':')
        # print('string= ', string)
        ip_host = string[1][string[1].rfind('//') + 2: len(string[1])]
        ip_port = string[2]
        if validateIp(ip_host, ip_port):
            print('%s 可用' % ip)
            validList.append(ip)
        else:
            print('%s 无效' % ip)
    return validList


# 验证ip是否有效
def validateIp(ip, port):
    try:
        telnetlib.Telnet(ip, port, timeout=3)
        print('%s 可用' % ip)
        return True
    except Exception as e:
        print('%s 无效' % ip)
        return False


def main():
    allProxys = []

    startPage = 1
    endPage = 5

    for index in range(startPage, endPage):
        print('查找第 %s 页的ip信息' % index)

        # 请求url，获取网页数据
        content = requestUrl(index)
        # 解析网页数据，获取ip和端口信息
        ip_list = parseProxyIpList(content)
        # ip_list = parseProxyIpList("")
        # 过滤有效的ip信息
        ip_list = filterValidProxyIp(ip_list)
        # 添加到有效列表中
        allProxys.append(ip_list)

        print('第 %s 页的有效ip有以下：' % index)
        print(ip_list)

        time.sleep(random.randint(10, 15))

    print('总共找到有效ip有以下：')
    print(allProxys)

    return allProxys


if __name__ == '__main__':
    proxies = main()
    # http://182.34.34.85:9999
    # validateIp("82.156.171.147", "3128")
